package com.mayy.spark.demo.dataset;

import lombok.Data;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Row;
import org.apache.spark.sql.SparkSession;

import java.io.Serializable;

/**
 * ./spark-submit --class  com.mayy.spark.demo.SimpleApp  --master spark://192.40.10.130:7077   /root/myspark/jars/spark-simple-demo-0.0.1.jar  /root/myspark/jars/people.json
 */
public class JavaSparkSQLExample {
    public static void main(String[] args) {

        String logFile;
        if (args != null && args.length > 0) {
            logFile = args[0];
        } else {
            logFile = "spark-official-demo/src/main/resources/people.json"; // 使用项目内的小文件
        }

        SparkSession sparkSession = SparkSession.builder()
                .appName("JavaSparkSQLExample")
                .config("spark.some.config.option", "some-value")
                .master("local[*]")
                .getOrCreate();
        runBasicDataFrameExample(sparkSession, logFile);

    }

    private static void runBasicDataFrameExample(SparkSession spark, String logFile) {
        Dataset<Row> df = spark.read().json(logFile);
        df.show();
        df.printSchema();
        df.select("name").show();
    }

    @Data
    public static class Person implements Serializable {
        private String name;
        private long age;
    }
}
